#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
东方财富基金排行数据爬取工具 - 最终版
功能：
1. 爬取基金排行榜数据
2. 支持翻页获取所有数据
3. 保存为Excel格式
4. 完整的错误处理和日志记录
5. 支持MySQL数据库存储
6. 自定义日志格式输出到log文件夹

"""

import urllib.request
import urllib.parse
import json
import csv
import time
import re
import os
import sys
from datetime import datetime, date
import logging
import http.cookiejar
import gzip
import io

# ==================== 配置常量 ====================
# 基金排行网站配置
FUND_CONFIG = {
    'base_url': 'https://fund.eastmoney.com',
    'api_url': 'http://fund.eastmoney.com/data/rankhandler.aspx',
    'page_size': 50,  # 每页显示的基金数量
    'max_pages': None,  # 最大爬取页数，None表示不限制
    'request_delay': 1.5,  # 请求间隔时间（秒）
}

# 请求头配置
HEADERS_CONFIG = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Cache-Control': 'max-age=0',
    'Referer': 'http://fund.eastmoney.com/data/fundranking.html',
    'Host': 'fund.eastmoney.com'
}

# API参数配置
API_PARAMS_CONFIG = {
    'op': 'ph',
    'dt': 'kf',
    'ft': 'all',
    'rs': '',
    'gs': '0',
    'sc': '1nzf',  # 按1年净值增长率排序
    'st': 'desc',  # 降序
    'qdii': '',
    'tabSubtype': ',,,,,',
    'dx': '1',
}

# 输出文件配置
OUTPUT_CONFIG = {
    'default_filename': 'fund_ranking_data.xlsx',
    'sheet_name': '基金排行数据',
    'include_timestamp': True,
}

# 导入项目配置
try:
    import sys
    import os
    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    from config import get_database_config
    DB_CONFIG = get_database_config()
except ImportError:
    # 备用配置
    DB_CONFIG = {
        'host': 'localhost',
        'user': 'root',
        'password': 'czya31505',
        'database': 'fund',
        'charset': 'utf8mb4'
    }

# 数据列映射
COLUMN_MAPPING = {
    0: '基金代码',
    1: '基金名称', 
    2: '基金简称',
    3: '日期',
    4: '单位净值',
    5: '累计净值',
    6: '日增长率(%)',
    7: '近1周(%)',
    8: '近1月(%)',
    9: '近3月(%)',
    10: '近6月(%)',
    11: '近1年(%)',
    12: '近2年(%)',
    13: '近3年(%)',
    14: '成立来(%)',
    15: '成立日期',
    16: '手续费'
}

# ==================== 自定义日志格式 ====================
class CustomFormatter(logging.Formatter):
    """自定义日志格式器：文件名-行数-级别-信息"""
    
    def format(self, record):
        # 获取文件名（不包含路径）
        filename = os.path.basename(record.pathname)
        # 获取行号
        lineno = record.lineno
        # 获取日志级别
        levelname = record.levelname
        # 获取日志信息
        message = record.getMessage()
        # 获取时间戳
        timestamp = self.formatTime(record, '%Y-%m-%d %H:%M:%S')
        
        # 格式：时间戳 文件名-行数-级别-信息
        return f"{timestamp} {filename}-{lineno}-{levelname}-{message}"

# 配置日志
def setup_logging():
    """设置日志配置"""
    # 确保log目录存在
    current_dir = os.path.dirname(os.path.abspath(__file__))
    project_root = os.path.dirname(os.path.dirname(current_dir))
    log_dir = os.path.join(project_root, 'log')
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    
    # 创建logger
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    
    # 清除已有的处理器
    logger.handlers.clear()
    
    # 创建文件处理器
    log_filename = os.path.join(log_dir, f'fund_scraper_{datetime.now().strftime("%Y%m%d")}.log')
    file_handler = logging.FileHandler(log_filename, encoding='utf-8')
    file_handler.setLevel(logging.INFO)
    
    # 创建控制台处理器
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.INFO)
    
    # 设置自定义格式器
    formatter = CustomFormatter()
    file_handler.setFormatter(formatter)
    console_handler.setFormatter(formatter)
    
    # 添加处理器到logger
    logger.addHandler(file_handler)
    logger.addHandler(console_handler)
    
    return logger

# 初始化日志
logger = setup_logging()

# Excel功能已移除

try:
    import pymysql
    PYMYSQL_AVAILABLE = True
except ImportError:
    PYMYSQL_AVAILABLE = False
    pymysql = None
    logger.warning("pymysql not available, database export will be disabled")

class FinalFundScraper:
    def __init__(self):
        self.base_url = FUND_CONFIG['api_url']
        self.main_page_url = 'http://fund.eastmoney.com/data/fundranking.html'
        self.all_data = []
        
        # 数据库配置
        self.db_config = DB_CONFIG
        
        # 日期配置
        self.start_date = '2024-09-17'  # 开始日期
        self.end_date = date.today().strftime('%Y-%m-%d')  # 结束日期，取当天日期
        
        # 创建cookie jar
        self.cookie_jar = http.cookiejar.CookieJar()
        
        # 创建opener
        self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cookie_jar))
        
        # 设置完整的请求头
        self.headers = HEADERS_CONFIG
    
    def init_session(self):
        """
        初始化会话，访问主页面获取cookies
        """
        try:
            logger.info("初始化会话，访问主页面...")
            req = urllib.request.Request(self.main_page_url, headers=self.headers)
            response = self.opener.open(req, timeout=30)
            
            # 处理gzip压缩
            raw_data = response.read()
            if response.info().get('Content-Encoding') == 'gzip':
                content = gzip.decompress(raw_data).decode('utf-8')
            else:
                content = raw_data.decode('utf-8')
                
            logger.info(f"主页面访问成功，状态码: {response.status}")
            logger.info(f"获得 {len(self.cookie_jar)} 个cookies")
            return True
        except Exception as e:
            logger.error(f"初始化会话失败: {e}")
            return False
    
    def get_fund_data(self, page=1, page_size=None):
        """
        获取指定页面的基金数据
        """
        if page_size is None:
            page_size = FUND_CONFIG['page_size']
            
        params = API_PARAMS_CONFIG.copy()
        params.update({
            'sd': self.start_date,  # 使用配置的开始日期
            'ed': self.end_date,    # 使用配置的结束日期
            'pi': str(page),
            'pn': str(page_size),
            'v': str(time.time())
        })
        
        try:
            # 构建URL
            url = self.base_url + '?' + urllib.parse.urlencode(params)
            
            # 创建请求
            req = urllib.request.Request(url, headers=self.headers)
            
            # 发送请求
            response = self.opener.open(req, timeout=30)
            
            # 处理gzip压缩
            raw_data = response.read()
            if response.info().get('Content-Encoding') == 'gzip':
                content = gzip.decompress(raw_data).decode('utf-8')
            else:
                content = raw_data.decode('utf-8')
            
            logger.info(f"第 {page} 页响应状态码: {response.status}，内容长度: {len(content)}")
            
            # 解析数据
            return self.parse_response(content)
            
        except Exception as e:
            logger.error(f"获取第 {page} 页数据失败: {e}")
            return []
    
    def parse_response(self, content):
        """
        解析API响应内容
        """
        try:
            # 查找数据部分
            if 'var rankData =' in content:
                # 提取rankData变量的值
                start = content.find('var rankData =') + len('var rankData =')
                end = content.find(';', start)
                data_str = content[start:end].strip()
                
                # 提取datas数组
                if 'datas:[' in data_str:
                    datas_start = data_str.find('datas:[') + len('datas:[')
                    datas_end = data_str.find('],', datas_start)
                    if datas_end == -1:
                        datas_end = data_str.find(']', datas_start)
                    
                    if datas_end > datas_start:
                        datas_content = data_str[datas_start:datas_end]
                        
                        # 分割每条记录，每条记录用引号包围
                        records = []
                        # 使用正则表达式匹配每条基金记录
                        pattern = r'"([^"]+)"'
                        matches = re.findall(pattern, datas_content)
                        
                        for match in matches:
                            # 每条记录是逗号分隔的字符串
                            fields = match.split(',')
                            if len(fields) >= 15:
                                record = {
                                    '基金代码': fields[0],
                                    '基金名称': fields[1],
                                    '基金简称': fields[2] if len(fields) > 2 else '',
                                    '日期': fields[3] if len(fields) > 3 else '',
                                    '单位净值': fields[4] if len(fields) > 4 else '',
                                    '累计净值': fields[5] if len(fields) > 5 else '',
                                    '日增长率(%)': fields[6] if len(fields) > 6 else '',
                                    '近1周(%)': fields[7] if len(fields) > 7 else '',
                                    '近1月(%)': fields[8] if len(fields) > 8 else '',
                                    '近3月(%)': fields[9] if len(fields) > 9 else '',
                                    '近6月(%)': fields[10] if len(fields) > 10 else '',
                                    '近1年(%)': fields[11] if len(fields) > 11 else '',
                                    '近2年(%)': fields[12] if len(fields) > 12 else '',
                                    '近3年(%)': fields[13] if len(fields) > 13 else '',
                                    '成立来(%)': fields[14] if len(fields) > 14 else '',
                                    '成立日期': fields[16] if len(fields) > 15 else '',
                                    '手续费': fields[19] if len(fields) > 19 else ''
                                }
                                records.append(record)
                        
                        return records
                    else:
                        logger.warning("datas 数组范围错误")
                        return []
                else:
                    logger.warning("未找到datas数组")
                    return []
            
            # 如果没有找到rankData，返回空列表
            logger.warning("未找到rankData变量")
            return []
            
        except Exception as e:
            logger.error(f"解析响应失败: {e}")
            import traceback
            logger.error(traceback.format_exc())
            return []
    
    def save_page_to_database(self, page_data):
        """
        将单页数据保存到数据库
        
        Args:
            page_data (list): 单页基金数据列表
            
        Returns:
            bool: 保存是否成功
        """
        if not page_data:
            return True
            
        if not PYMYSQL_AVAILABLE:
            logger.warning("pymysql不可用，跳过数据库保存")
            return False
            
        connection = None
        try:
            connection = self.get_db_connection()
            if not connection:
                return False
                
            with connection.cursor() as cursor:
                # 创建表（如果不存在）
                create_table_sql = """
                CREATE TABLE IF NOT EXISTS final_fund_scraper (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    fund_code VARCHAR(20) NOT NULL,
                    fund_name VARCHAR(200),
                    fund_short_name VARCHAR(100),
                    nav_date DATE,
                    unit_nav DECIMAL(10,4),
                    cumulative_nav DECIMAL(10,4),
                    daily_growth_rate DECIMAL(8,4),
                    weekly_return DECIMAL(8,4),
                    monthly_return DECIMAL(8,4),
                    quarterly_return DECIMAL(8,4),
                    half_year_return DECIMAL(8,4),
                    yearly_return DECIMAL(8,4),
                    two_year_return DECIMAL(8,4),
                    three_year_return DECIMAL(8,4),
                    since_inception_return DECIMAL(8,4),
                    inception_date DATE,
                    management_fee VARCHAR(20),
                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    update_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                    UNIQUE KEY unique_fund_date (fund_code, nav_date)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
                """
                cursor.execute(create_table_sql)
                
                # 插入数据的SQL语句（使用ON DUPLICATE KEY UPDATE避免重复）
                insert_sql = """
                INSERT INTO final_fund_scraper (
                    fund_code, fund_name, fund_short_name, nav_date, unit_nav, cumulative_nav,
                    daily_growth_rate, weekly_return, monthly_return, quarterly_return,
                    half_year_return, yearly_return, two_year_return, three_year_return,
                    since_inception_return, inception_date, management_fee
                ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                ON DUPLICATE KEY UPDATE
                    fund_name = VALUES(fund_name),
                    fund_short_name = VALUES(fund_short_name),
                    unit_nav = VALUES(unit_nav),
                    cumulative_nav = VALUES(cumulative_nav),
                    daily_growth_rate = VALUES(daily_growth_rate),
                    weekly_return = VALUES(weekly_return),
                    monthly_return = VALUES(monthly_return),
                    quarterly_return = VALUES(quarterly_return),
                    half_year_return = VALUES(half_year_return),
                    yearly_return = VALUES(yearly_return),
                    two_year_return = VALUES(two_year_return),
                    three_year_return = VALUES(three_year_return),
                    since_inception_return = VALUES(since_inception_return),
                    inception_date = VALUES(inception_date),
                    management_fee = VALUES(management_fee),
                    update_date = CURRENT_TIMESTAMP
                """
                
                success_count = 0
                error_count = 0
                
                for record in page_data:
                    try:
                        def convert_to_float(value):
                            if value is None or value == '' or value == '--':
                                return None
                            try:
                                # 移除百分号并转换为浮点数
                                if isinstance(value, str) and value.endswith('%'):
                                    return float(value[:-1])
                                return float(value)
                            except (ValueError, TypeError):
                                return None
                        
                        def convert_to_date(date_str):
                            if date_str is None or date_str == '' or date_str == '--':
                                return None
                            try:
                                # 尝试解析日期格式
                                if len(date_str) == 10 and '-' in date_str:
                                    return date_str
                                elif len(date_str) == 8:
                                    # 格式：20240917
                                    return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}"
                                else:
                                    return None
                            except:
                                return None
                        
                        # 准备数据
                        data = (
                            record.get('基金代码', ''),
                            record.get('基金名称', ''),
                            record.get('基金简称', ''),
                            convert_to_date(record.get('日期', '')) or self.start_date,
                            convert_to_float(record.get('单位净值', '')),
                            convert_to_float(record.get('累计净值', '')),
                            convert_to_float(record.get('日增长率(%)', '')),
                            convert_to_float(record.get('近1周(%)', '')),
                            convert_to_float(record.get('近1月(%)', '')),
                            convert_to_float(record.get('近3月(%)', '')),
                            convert_to_float(record.get('近6月(%)', '')),
                            convert_to_float(record.get('近1年(%)', '')),
                            convert_to_float(record.get('近2年(%)', '')),
                            convert_to_float(record.get('近3年(%)', '')),
                            convert_to_float(record.get('成立来(%)', '')),
                            convert_to_date(record.get('成立日期', '')),
                            record.get('手续费', '')
                        )
                        
                        cursor.execute(insert_sql, data)
                        success_count += 1
                        
                    except Exception as e:
                        error_count += 1
                        logger.warning(f"插入记录失败: {record.get('基金代码', 'Unknown')}, 错误: {e}")
                
                # 提交事务
                connection.commit()
                
                logger.info(f"页面数据保存完成: 成功 {success_count} 条，失败 {error_count} 条")
                return True
                
        except Exception as e:
            logger.error(f"数据库操作失败: {e}")
            try:
                if connection:
                    connection.rollback()
            except:
                pass
            return False
        finally:
            try:
                if connection:
                    connection.close()
            except:
                pass
    
    def scrape_all_pages(self, max_pages=None):
        """
        爬取所有页面数据，每页数据立即写入数据库
        """
        logger.info("开始爬取基金数据...")
        
        # 初始化会话
        if not self.init_session():
            logger.error("会话初始化失败，停止爬取")
            return []
        
        if max_pages is None:
            max_pages = FUND_CONFIG['max_pages']
            
        page = 1
        total_records = 0
        consecutive_empty_pages = 0
        
        while max_pages is None or page <= max_pages:
            logger.info(f"正在爬取第 {page} 页...")
            
            records = self.get_fund_data(page=page)
            
            if not records:
                consecutive_empty_pages += 1
                logger.warning(f"第 {page} 页无数据")
                
                # 如果连续3页都没有数据，停止爬取
                if consecutive_empty_pages >= 3:
                    logger.info("连续多页无数据，停止爬取")
                    break
            else:
                consecutive_empty_pages = 0
                self.all_data.extend(records)
                total_records += len(records)
                logger.info(f"第 {page} 页获取到 {len(records)} 条数据，累计 {total_records} 条")
                
                # 立即将当前页数据保存到数据库
                if self.save_page_to_database(records):
                    logger.info(f"第 {page} 页数据已保存到数据库")
                else:
                    logger.error(f"第 {page} 页数据保存到数据库失败")
            
            page += 1
            
            # 添加延时，避免请求过于频繁
            time.sleep(FUND_CONFIG['request_delay'])
        
        logger.info(f"爬取完成，共获取 {total_records} 条基金数据")
        return self.all_data
    
    # Excel保存功能已移除
    
    # CSV保存功能已移除
    
    def get_fund_data_for_api(self, page=1, page_size=50, nav_date=None, sort_by='daily_growth_rate', order='desc', fund_name=None):
        """
        为API接口提供基金数据查询功能
        
        Args:
            page: 页码，默认为1
            page_size: 每页记录数，默认为50
            nav_date: 净值日期，格式为YYYY-MM-DD
            sort_by: 排序字段
            order: 排序方向 (asc, desc)
            fund_name: 基金名称，支持模糊查询
            
        Returns:
            dict: 包含数据和总数的字典
        """
        try:
            conn = self.get_db_connection()
            if conn is None:
                logger.error("数据库连接失败")
                return {
                    'data': [],
                    'total': 0,
                    'count': 0,
                    'error': '数据库连接失败'
                }
                
            cursor = conn.cursor()
            
            # 构建查询条件
            where_conditions = []
            params = []
            
            if nav_date:
                where_conditions.append("nav_date = %s")
                params.append(nav_date)
                
            if fund_name:
                where_conditions.append("fund_name LIKE %s")
                params.append(f"%{fund_name}%")
            
            # 构建基础查询
            base_query = "SELECT * FROM final_fund_scraper"
            if where_conditions:
                base_query += " WHERE " + " AND ".join(where_conditions)
            
            # 获取总数
            count_query = f"SELECT COUNT(*) FROM ({base_query}) as count_table"
            cursor.execute(count_query, params)
            result = cursor.fetchone()
            total_count = result[0] if result else 0
            
            # 构建排序
            valid_sort_fields = [
                'fund_code', 'fund_name', 'nav_date', 'unit_nav', 'cumulative_nav',
                'daily_growth_rate', 'weekly_return', 'monthly_return', 'quarterly_return',
                'half_year_return', 'yearly_return', 'two_year_return', 'three_year_return',
                'since_inception_return', 'inception_date', 'management_fee'
            ]
            
            if sort_by in valid_sort_fields:
                order_direction = 'DESC' if order == 'desc' else 'ASC'
                order_clause = f" ORDER BY {sort_by} {order_direction}, fund_name ASC"
            else:
                order_clause = " ORDER BY nav_date DESC, fund_name ASC"
            
            # 计算分页
            offset = (page - 1) * page_size
            total_pages = (total_count + page_size - 1) // page_size
            
            # 添加排序和分页
            data_query = base_query + order_clause + " LIMIT %s OFFSET %s"
            params.extend([page_size, offset])
            
            # 执行数据查询
            cursor.execute(data_query, params)
            columns = [desc[0] for desc in cursor.description]
            rows = cursor.fetchall()
            
            # 转换为字典列表
            data = []
            for row in rows:
                record = {}
                for i, value in enumerate(row):
                    column_name = columns[i]
                    # 处理日期类型
                    if isinstance(value, datetime):
                        record[column_name] = value.strftime('%Y-%m-%d')
                    elif value is None:
                        record[column_name] = ''
                    else:
                        record[column_name] = value
                data.append(record)
            
            cursor.close()
            conn.close()
            
            return {
                'data': data,
                'total': total_count,
                'count': len(data),
                'pagination': {
                    'current_page': page,
                    'page_size': page_size,
                    'total_pages': total_pages,
                    'has_next': page < total_pages,
                    'has_previous': page > 1
                }
            }
            
        except Exception as e:
            logger.error(f"API数据查询失败: {e}")
            return {
                'data': [],
                'total': 0,
                'count': 0,
                'error': str(e)
            }
    
    def get_available_dates(self):
        """
        获取数据库中可用的净值日期列表
        
        Returns:
            list: 可用日期列表，按降序排列
        """
        try:
            conn = self.get_db_connection()
            if conn is None:
                logger.error("数据库连接失败")
                return []
                
            cursor = conn.cursor()
            
            query = "SELECT DISTINCT nav_date FROM final_fund_scraper ORDER BY nav_date DESC"
            cursor.execute(query)
            dates = [row[0].strftime('%Y-%m-%d') if isinstance(row[0], datetime) else str(row[0]) for row in cursor.fetchall()]
            
            cursor.close()
            conn.close()
            
            return dates
            
        except Exception as e:
            logger.error(f"获取可用日期失败: {e}")
            return []
    
    def get_fund_ranking_statistics(self, nav_date=None):
        """
        获取基金排行统计信息
        """
        try:
            connection = self.get_db_connection()
            if connection is None:
                logger.error("数据库连接失败")
                return {
                    'total_count': 0,
                    'avg_daily_growth_rate': 0,
                    'avg_yearly_return': 0,
                    'avg_unit_nav': 0
                }
            
            cursor = connection.cursor()
            
            # 构建查询条件
            where_clause = ""
            params = []
            if nav_date:
                where_clause = "WHERE nav_date = %s"
                params.append(nav_date)
            else:
                # 使用最新日期
                where_clause = "WHERE nav_date = (SELECT MAX(nav_date) FROM final_fund_scraper)"
            
            # 查询统计信息
            query = f"""
            SELECT 
                COUNT(*) as total_count,
                AVG(daily_growth_rate) as avg_daily_growth_rate,
                AVG(yearly_return) as avg_yearly_return,
                AVG(unit_nav) as avg_unit_nav
            FROM final_fund_scraper 
            {where_clause}
            AND daily_growth_rate IS NOT NULL 
            AND yearly_return IS NOT NULL 
            AND unit_nav IS NOT NULL
            """
            
            cursor.execute(query, params)
            result = cursor.fetchone()
            
            cursor.close()
            connection.close()
            
            if result:
                return {
                    'total_count': int(result[0]) if result[0] else 0,
                    'avg_daily_growth_rate': round(float(result[1]), 2) if result[1] else 0,
                    'avg_yearly_return': round(float(result[2]), 2) if result[2] else 0,
                    'avg_unit_nav': round(float(result[3]), 4) if result[3] else 0
                }
            else:
                return {
                    'total_count': 0,
                    'avg_daily_growth_rate': 0,
                    'avg_yearly_return': 0,
                    'avg_unit_nav': 0
                }
                
        except Exception as e:
            logger.error(f"获取统计信息失败: {e}")
            return {
                'total_count': 0,
                'avg_daily_growth_rate': 0,
                'avg_yearly_return': 0,
                'avg_unit_nav': 0
            }
    
    def get_db_connection(self):
        """
        获取数据库连接
        """
        if not PYMYSQL_AVAILABLE:
            logger.error("pymysql库未安装，无法连接数据库")
            return None
            
        if not PYMYSQL_AVAILABLE or pymysql is None:
            logger.error("pymysql库未安装，无法连接数据库")
            return None
            
        try:
            connection = pymysql.connect(
                host=self.db_config['host'],
                user=self.db_config['user'],
                password=self.db_config['password'],
                database=self.db_config['database'],
                charset=self.db_config['charset']
            )
            return connection
        except Exception as e:
            logger.error(f"数据库连接失败: {e}")
            return None
    
    def save_to_database(self):
        """
        保存数据到MySQL数据库
        """
        if not PYMYSQL_AVAILABLE:
            logger.error("pymysql库未安装，无法保存到数据库")
            return False
            
        if not self.all_data:
            logger.warning("没有数据可保存")
            return False
        
        connection = self.get_db_connection()
        if not connection:
            return False
        
        try:
            with connection.cursor() as cursor:
                # 准备插入语句（使用ON DUPLICATE KEY UPDATE确保唯一性）
                insert_sql = """
                INSERT INTO final_fund_scraper (
                    fund_code, fund_name, fund_short_name, nav_date, unit_nav, cumulative_nav,
                    daily_growth_rate, weekly_return, monthly_return, quarterly_return,
                    half_year_return, yearly_return, two_year_return, three_year_return,
                    since_inception_return, inception_date, management_fee
                ) VALUES (
                    %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
                ) ON DUPLICATE KEY UPDATE
                    fund_name = VALUES(fund_name),
                    fund_short_name = VALUES(fund_short_name),
                    unit_nav = VALUES(unit_nav),
                    cumulative_nav = VALUES(cumulative_nav),
                    daily_growth_rate = VALUES(daily_growth_rate),
                    weekly_return = VALUES(weekly_return),
                    monthly_return = VALUES(monthly_return),
                    quarterly_return = VALUES(quarterly_return),
                    half_year_return = VALUES(half_year_return),
                    yearly_return = VALUES(yearly_return),
                    two_year_return = VALUES(two_year_return),
                    three_year_return = VALUES(three_year_return),
                    since_inception_return = VALUES(since_inception_return),
                    inception_date = VALUES(inception_date),
                    management_fee = VALUES(management_fee),
                    update_date = CURRENT_TIMESTAMP
                """
                
                success_count = 0
                error_count = 0
                
                for record in self.all_data:
                    try:
                        # 数据转换和清理
                        def convert_to_float(value):
                            if value is None or value == '' or value == '--':
                                return None
                            try:
                                return float(value)
                            except (ValueError, TypeError):
                                return None
                        
                        def convert_to_date(date_str):
                            if date_str is None or date_str == '' or date_str == '--':
                                return None
                            try:
                                # 尝试解析日期格式
                                if len(date_str) == 10 and '-' in date_str:
                                    return date_str
                                elif len(date_str) == 8:
                                    # 格式：20240917
                                    return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}"
                                else:
                                    return None
                            except:
                                return None
                        
                        # 准备数据
                        data = (
                            record.get('基金代码', ''),
                            record.get('基金名称', ''),
                            record.get('基金简称', ''),
                            convert_to_date(record.get('日期', '')) or self.start_date,
                            convert_to_float(record.get('单位净值', '')),
                            convert_to_float(record.get('累计净值', '')),
                            convert_to_float(record.get('日增长率(%)', '')),
                            convert_to_float(record.get('近1周(%)', '')),
                            convert_to_float(record.get('近1月(%)', '')),
                            convert_to_float(record.get('近3月(%)', '')),
                            convert_to_float(record.get('近6月(%)', '')),
                            convert_to_float(record.get('近1年(%)', '')),
                            convert_to_float(record.get('近2年(%)', '')),
                            convert_to_float(record.get('近3年(%)', '')),
                            convert_to_float(record.get('成立来(%)', '')),
                            convert_to_date(record.get('成立日期', '')),
                            record.get('手续费', '')
                        )
                        
                        cursor.execute(insert_sql, data)
                        success_count += 1
                        
                    except Exception as e:
                        error_count += 1
                        logger.warning(f"插入记录失败: {record.get('基金代码', 'Unknown')}, 错误: {e}")
                
                # 提交事务
                connection.commit()
                
                logger.info(f"数据库保存完成: 成功 {success_count} 条，失败 {error_count} 条")
                return True
                
        except Exception as e:
            logger.error(f"数据库操作失败: {e}")
            connection.rollback()
            return False
        finally:
            connection.close()
    
    def print_summary(self):
        """
        打印数据摘要
        """
        if not self.all_data:
            logger.info("没有数据可显示")
            return
        
        logger.info(f"\n=== 数据摘要 ===")
        logger.info(f"总记录数: {len(self.all_data)}")
        
        # 显示前5条数据
        logger.info(f"\n前5条数据:")
        for i, record in enumerate(self.all_data[:5]):
            logger.info(f"第 {i+1} 条: {record['基金代码']} - {record['基金名称']} - 近1年: {record['近1年(%)']}%")
        
        # 统计信息
        if len(self.all_data) > 0:
            try:
                # 统计近1年收益率分布
                returns_1y = []
                for record in self.all_data:
                    try:
                        ret = record['近1年(%)']
                        if ret and ret != '--':
                            returns_1y.append(float(ret))
                    except (ValueError, TypeError):
                        pass
                
                if returns_1y:
                    logger.info(f"\n近1年收益率统计:")
                    logger.info(f"最高收益率: {max(returns_1y):.2f}%")
                    logger.info(f"最低收益率: {min(returns_1y):.2f}%")
                    logger.info(f"平均收益率: {sum(returns_1y)/len(returns_1y):.2f}%")
                    
                    # 收益率分布
                    positive_count = len([r for r in returns_1y if r > 0])
                    logger.info(f"正收益基金数量: {positive_count} ({positive_count/len(returns_1y)*100:.1f}%)")
                    
            except Exception as e:
                logger.warning(f"统计信息计算失败: {e}")
    
    def run(self, max_pages=None, save_excel=True, save_csv=False, save_database=True):
        """
        运行爬虫
        """
        try:
            # 爬取数据
            self.scrape_all_pages(max_pages=max_pages)
            
            # 显示摘要
            self.print_summary()
            
            # 保存数据
            if self.all_data:
                if save_database:
                    self.save_to_database()
                # CSV保存功能已移除
            else:
                logger.warning("未获取到任何数据")
                
        except Exception as e:
            logger.error(f"爬虫运行失败: {e}")
            import traceback
            logger.error(traceback.format_exc())

def main():
    """
    主函数
    """
    logger.info("=== 东方财富基金排行数据爬取工具 ===")
    logger.info("目标网站: https://fund.eastmoney.com/data/fundranking.html")
    logger.info("功能: 爬取基金排行数据并保存为Excel文件")
    
    scraper = FinalFundScraper()
    
    # 运行爬虫，爬取所有页面数据
    scraper.run(max_pages=None, save_excel=True, save_csv=True, save_database=True)
    
    logger.info("\n程序执行完成！")
    logger.info("数据已保存为数据库、Excel和CSV格式")

if __name__ == '__main__':
    main()